import java.io.File;
import java.io.InputStream;
import java.util.Date;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.w3c.dom.*;

import com.mysql.jdbc.Connection;

public class DOMJavaParser {
	DBConnection dbconn = new DBConnection();
	String url = "http://de.wikipedia.org//w/api.php?action=query&prop=info&format=xml&generator=allpages&gaplimit=max&apdir=ascending&gapcontinue=";

	int counter = 0;
	String gapcontinue = "";
	boolean myboolean;

	DOMJavaParser() {

	}

	public void parsePage() {
		// &&counter < 150000
		boolean iscontinue = true;
		while (iscontinue) {
			// document holen
			HttpConnection connection = new HttpConnection(url + gapcontinue);
			Document mydoc = connection.returnXML();
			// while schleife anpassen

			NodeList nodelist = mydoc.getElementsByTagName("allpages");
			Element myel = (Element) nodelist.item(0);
			gapcontinue = myel.getAttribute("gapcontinue");

			// Element el = mydoc.getElementById("allpages");
			// gapcontinue = el.getAttribute("gapcontinue");

			// auswerten
			NodeList nodes = mydoc.getElementsByTagName("page");

			String[][] values = new String[nodes.getLength()][3];
			for (int i = 0; i < nodes.getLength(); i++) {

				Node node = nodes.item(i);
				if (node.getNodeType() == Node.ELEMENT_NODE) {
					Element element = (Element) node;
					if (element.hasAttribute("redirect")) {
						System.out.println("redirect <page >"
								+ String.valueOf(i));

					} else {
						Element ele = (Element) node;
						String page_id = ele.getAttribute("pageid");
						String titel = ele.getAttribute("title");
						if (titel.length() > 30)
							titel = titel.substring(0, 30);
						String touched = ele.getAttribute("touched");

						touched = touched.replace('T', ' ');
						touched = touched.replace('Z', ' ');
						touched = touched.substring(0, 19);
						values[i][0] = page_id;
						values[i][1] = titel;
						values[i][2] = touched;
					}

				}

			}
			String[][] helperarray = new String[values.length][3];

			int counter = 0;
			for (int n = 0; n < values.length; n++) {

				if (values[n][0] != null) {

					helperarray[counter][0] = values[n][0];
					helperarray[counter][1] = values[n][1];
					helperarray[counter][2] = values[n][2];
					counter++;

				}
			}
			for (int n = 0; n < helperarray.length; n++) {
				if (helperarray[n][0] == null) {
					counter = n;
					System.out
							.println("counterlength ist " + String.valueOf(n));
					break;
				}
			}
			String[][] colArray = new String[counter][3];
			for (int n = 0; n < counter; n++) {
				colArray[n][0] = helperarray[n][0];
				colArray[n][1] = helperarray[n][1];
				colArray[n][2] = helperarray[n][2];
			}

			// pages in db schreiben

			 dbconn.commitOnDB(colArray, dbconn.getPage_sql());

			// parse revisions
			String[] myarray = new String[colArray.length];
			for (int row = 0; row < colArray.length; row++) {
				myarray[row] = colArray[row][0];
			}

			parseRevision(myarray);

		}

	}

	public void parseRevision(String[] pages) {
		String[] mypages = pages;
		boolean iscontinue;
		String mycontinue;
		for (int k = 0; k < mypages.length; k++) {
			iscontinue = true;
			mycontinue = "";
			while (iscontinue) {
				String mystring = "http://de.wikipedia.org/w/api.php?format=xml&action=query&pageids="
						+ mypages[k]
						+ "&prop=revisions&rvprop=ids|flags|timestamp|user|userid&rvlimit=max"
						+ mycontinue + "&continue=";
				HttpConnection con = new HttpConnection(mystring);
				Document mydoc = con.returnXML();

				System.out.println("letztes Kind: "
						+ mydoc.getDocumentElement().getLastChild()
								.getNodeName().equals("continue"));
				if (mydoc.getDocumentElement().getLastChild().getNodeName()
						.equals("continue")) {
					NodeList cont = mydoc.getElementsByTagName("continue");
					Element cont_ele = (Element) cont.item(0);
					mycontinue = "&rvcontinue="
							+ cont_ele.getAttribute("rvcontinue");
					System.out.println(mystring);
				}

				else {
					iscontinue = false;
				}

				// verarbeite rev nodes
				NodeList nodes = mydoc.getElementsByTagName("rev");
				String[][] values = new String[nodes.getLength()][5];
				String[][] pid_rid = new String[nodes.getLength()][2];
				for (int i = 0; i < nodes.getLength(); i++) {
					Node node = nodes.item(i);

						Element ele = (Element) node;
						// revision_id, timestamp, user, user_id, minor
						String minor;
						if (ele.hasAttribute("minor")) {
							minor = "1";
						}

						else
							minor = "0";
						String revid = ele.getAttribute("revid");
						String timestamp = ele.getAttribute("timestamp");
						String user = ele.getAttribute("user");
						if (user.length() > 30) {
							user = user.substring(0, 30);
						}
						String userid = ele.getAttribute("userid");

						timestamp = timestamp.replace('T', ' ');
						timestamp = timestamp.replace('Z', ' ');
						timestamp = timestamp.substring(0, 19);

						values[i][0] = revid;
						values[i][1] = timestamp;
						values[i][2] = user;
						values[i][3] = userid;
						values[i][4] = minor;

						pid_rid[i][0] = mypages[k];
						pid_rid[i][1] = revid;
					}
						dbconn.commitOnDB(values, dbconn.getRevision_sql());
						dbconn.commitOnDB(pid_rid,dbconn.getPage_revision_sql());

					
				}
			}
		}
	}

// boolean iscontinue = true;
// int page = 0;
// while (iscontinue && page < mypages.length) {
// System.out.println("Remaining pages to get revisions"
// + String.valueOf(mypages.length - page));
// iscontinue = true;
// System.out
// .println("                                                       "
// + mypages[page]);
// String mystring =
// "http://de.wikipedia.org/w/api.php?format=xml&action=query&pageids="
// + mypages[page]
// + "&prop=revisions&rvprop=ids|flags|timestamp|user|userid&rvlimit=max&"
// + mycontinue + "&continue=";
// HttpConnection con = new HttpConnection(mystring);
// Document mydoc = con.returnXML();
//
// Element root = mydoc.getDocumentElement();
// NodeList nodelist = root.getChildNodes();
// for (int index = 0; index < nodelist.getLength(); index++) {
// Node node = nodelist.item(index);
// Element element = (Element) node;
//
// String s = element.getNodeName();
// if (s == "continue") {
//
// mycontinue = "&rvcontinue="+ element.getAttribute("rvcontinue");
// System.out
// .println("                                                                rvcontinue  "
// + mycontinue);
// } else
// page++;
// }
//
// // verarbeite rev nodes
// NodeList nodes = mydoc.getElementsByTagName("rev");
// String[][] values = new String[nodes.getLength()][5];
// String[][] pid_rid = new String[nodes.getLength()][2];
// for (int i = 0; i < nodes.getLength(); i++) {
// Node node = nodes.item(i);
//
// if (node.getNodeType() == Node.ELEMENT_NODE) {
// Element ele = (Element) node;
// // revision_id, timestamp, user, user_id, minor
// String minor;
// if (ele.hasAttribute("minor")) {
// minor = "1";
// }
//
// else
// minor = "0";
// String revid = ele.getAttribute("revid");
// String timestamp = ele.getAttribute("timestamp");
// String user = ele.getAttribute("user");
// if (user.length() > 30) {
// user = user.substring(0, 30);
// }
// String userid = ele.getAttribute("userid");
//
// timestamp = timestamp.replace('T', ' ');
// timestamp = timestamp.replace('Z', ' ');
// timestamp = timestamp.substring(0, 19);
//
// values[i][0] = revid;
// values[i][1] = timestamp;
// values[i][2] = user;
// values[i][3] = userid;
// values[i][4] = minor;
//
// pid_rid[i][0] = mypages[page];
// pid_rid[i][1] = revid;
//
// }
// }
// dbconn.commitOnDB(values, dbconn.getRevision_sql());
// dbconn.commitOnDB(pid_rid, dbconn.getPage_revision_sql());
//
// }

